import json
import os
import sys
import logging
import requests
import threading

from multiprocessing import Pool, Manager, Lock
from datetime import datetime
from collections import defaultdict
from tqdm import tqdm
from concurrent.futures import ThreadPoolExecutor, as_completed

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))


from utils.time_trans import timestamp_trans
from config import *

# CRITICAL > ERROR > WARNING > INFO > DEBUG > NOTSET
logging.basicConfig(filename=os.path.join(LOG_PATH, "ioda_spider.log"), level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')


class IODA_SPIDER():
    def __init__(self) -> None:
        """
            Initializes the class instance with default values for start_time, end_time, events_data, and signal_data.
        """
        self.start_time: int = None
        self.end_time: int = None
        self.events_data :dict = defaultdict(list)
        self.signal_data :dict = defaultdict(list)

    def set_config(self, start_time:None, end_time:None):
        """
            The function to set the config of IODA.

            start_time (int): timestamp of start time.
            end_time (int): timestamp of end time.
            entityType (str, optional): None, continent, country, region, county, asn. Defaults to None.
            entityCode (str, optional): the code of entityType, such as: US, CN. Defaults to None.
            includeAlerts (bool, optional): Whether to include a                                            lert events. Defaults to False.
            limit (int, optional): Maximum number of events to return. Defaults to 2000.
            overall (bool, optional): Merge events across different data sources. Defaults to False.
            datasource (str): the data source of the signal. e.g. None, bgp, ucsd-nt, ping-slash24, merit-nt, gtr, gtr-norm. Default is None.
        """
        try:
            with open(CONFIG_FILE_PATH, "r") as f:
                config = json.load(f)
        except FileNotFoundError:
            print("找不到配置文件!")

        self.start_time = int(start_time.timestamp()) if start_time else None
        self.end_time = int(end_time.timestamp()) if start_time else None
        try:
            # The attributes following are events attributes.
            self.entityType: str = config["entityType"]
            self.entityCode: str = config["entityCode"]
            self.includeAlerts = config["includeAlerts"]
            self.limit:int = config["limit"]
            self.overall:bool = config["overall"]
            
            # The attributes following are signal attributes.
            # self.entityType2: str = config["entityType2"]
            # self.entityCode2: str = config["entityCode2"]
            self.datasource:str = config["datasource"]

        except KeyError:
            logging.error("请检查配置文件是否正确,缺少/错误的键值!")


    def run(self, type: str):
        """
            The function to run the IODA_SPIDER.

        Args:
            type (str): 'events' or 'signal'.
        """
        start_time = self.start_time 
        end_time = self.end_time
        tag = 0
        if type == "events":
            while end_time > start_time:
                if tag % 10 == 0:
                    print(f"Has been crawled {timestamp_trans(start_time)}.")
                if end_time < start_time + 7*24*3600:
                    print(f"Has been crawled {timestamp_trans(start_time)} to {timestamp_trans(end_time)}.")
                    self.events_spider(start_time, end_time)
                    break

                self.events_spider(start_time, start_time + 7*24*3600)
                start_time += 7*24*3600
                tag += 1

        elif type == "signal":
            while end_time > start_time:
                if end_time < start_time + 7*24*3600:
                    self.signal_spider()
                    break

                self.signal_spider()
                start_time += 7*24*3600
        
        else:
            logging.error("Invalid type, Please input 'events' or 'signal'")
        
        self.__save(type)

    def __save(self, type: str):
        if type == "events":
            with open(os.path.join(OUTPUT_PATH, "ioda_events_data.json"), "w") as f:
                json.dump(self.events_data, f)
        elif type == "signal":
            with open(os.path.join(OUTPUT_PATH, f"ioda_signal_data.json"), "w") as f:
                json.dump(self.signal_data, f)

    def process_event(self, item, queue):
        entityType, entityCode= item["location"].split("/")
        start = item["start"]
        duration = item["duration"]
        start_time = start - duration
        end_time = start + duration
        data = self.signal_spider(start_time, end_time, entityType, entityCode)
        queue.put(data)

    def write_from_queue(self, queue, filename):
        list_temp = []
        path = os.path.join(OUTPUT_PATH,filename)
        with open(path, 'w') as f:
            while not queue.empty():
                list_temp.append(queue.get())
            json.dump(list_temp, f, ensure_ascii=False)

    def load_events_data(self, path:str):
        path = os.path.join(OUTPUT_PATH,path) if path else os.path.join(OUTPUT_PATH, "ioda_events_data.json")
        with open(path, "r") as f:
            self.events_data = json.load(f)

        subname = os.path.basename(path).split('_')[-1]

        events_data = self.events_data
        manager = Manager()
        queue = manager.Queue()  

        pool = Pool(processes=32)

        pbar = tqdm(total=len(events_data), unit="event")

        for data in events_data:
            pool.apply_async(self.process_event,(data, queue), callback = lambda _: pbar.update(1))

        pool.close()
        pool.join()


        self.write_from_queue(queue, "ioda_signal_"+subname)
        # with Pool(processes=16) as pool:
        #     for _ in tqdm(pool.imap_unordered(self.process_event, events_data), total=len(events_data)):
        #         pass

        #     pool.close()
        #     pool.join()

        # self.signal_data = dict(self.shared_events_data)

        # self.__save("signal")

    def events_spider(self, start_time:int, end_time:int):
        """
            The function to get outages events from IODA.(API https://api.ioda.inetintel.cc.gatech.edu/v2/outages/events)

            start_time (int): timestamp of start time.
            end_time (int): timestamp of end time.
            entityType (str, optional): None, continent, country, region, county, asn. Defaults to None.
            entityCode (str, optional): the code of entityType, such as: US, CN. Defaults to None.
            includeAlerts (bool, optional): Whether to include alert events. Defaults to False.
            limit (int, optional): Maximum number of events to return. Defaults to 2000.
            overall (bool, optional): Merge events across different data sources. Defaults to False.
        """

        url_ = "https://api.ioda.inetintel.cc.gatech.edu/v2/outages/events?"
        entityType_ = f"entityType={self.entityType}&" if self.entityType else ""
        entityCode_ = f"entityCode={self.entityCode}&" if self.entityCode else ""
        time_ = f"from={start_time}&until={end_time}&"
        alerts_ = f"includeAlerts={self.includeAlerts}&" 
        limit_ = f"limit={self.limit}&" if self.limit else ""
        overall_ = f"overall={self.overall}"

        url = url_ + entityType_ + entityCode_ + time_ + alerts_ + limit_ + overall_
        try:
            response = requests.request('GET', url)
        except Exception as e:
            logging.error(e)
            return None
        
        if response.status_code == 200: 
            content = response.json()
            self.events_data["data"] += content['data']
        else:
            i = 0
            while response.status_code != 200:
                print("进行第{}次重试:".format(i:=i+1), response.status_code)
                if i>5 :
                    print("请求失败:", response.status_code)
                    log_info = "status_code:{} entityType:{} entityCode:{} from:{} to:{} includeAlerts:{} limit:{}".format(
                        response.status_code, self.entityType, self.entityCode,
                        start_time, end_time, self.includeAlerts, self.limit
                    )
                    logging.warning(log_info)
                    return None
                response = requests.request('GET', url)
                if response.status_code == 200:
                    content = response.json()
                    self.events_data["data"] += content['data']
                    break

        # else:
        #     print("请求失败:", response.status_code)
        #     log_info = "status_code:{} entityType:{} entityCode:{} from:{} to:{} includeAlerts:{} limit:{}".format(
        #         response.status_code, self.entityType, self.entityCode,
        #         start_time, end_time, self.includeAlerts, self.limit
        #         )
            
        #     logging.warning(log_info)
        #     return None

    # FIXME: The function is not complete, need to be improved.
    # 1. The scope of time is not correct.
    # 2. The save of data is not suitable.
    def signal_spider(self, start_time: int, end_time: int, entityType: str, entityCode: str):
        """
            The function to get raw signals from IODA.(API https://api.ioda.inetintel.cc.gatech.edu/v2/signals/raw/)

        Args:
            start_time (int): timestamp of events begin.
            end_time (int): timestamp of events end.
            entityType (str): continent, country, region, county, asn.
            entityCode (str): the code of entityType, such as: US, CN, 174.
        """

        if entityCode is None or entityType is None:
            print("Please input entityType and entityCode!")
            return None
        
        url_ = "https://api.ioda.inetintel.cc.gatech.edu/v2/signals/raw/"
        entityType_ = f"{entityType}/"
        entityCode_ = f"{entityCode}?"

        # FIXME: the time format is not correct
        time = f"from={start_time}&until={end_time}"
        
        datasource_ = f"&datasource={self.datasource}" if self.datasource else ""
        
        url = url_ + entityType_ + entityCode_ + time + datasource_

        response = requests.request('GET', url)
        if response.status_code == 200:
            content = response.json()
            
            return content
            # self.signal_data["data"].append(content)
        else:
            print("请求失败:", response.status_code)
            logging.info("请求失败:", response.status_code)
            print("请求url:", url)
            return None


if __name__ == "__main__":
    """test the IODA_SPIDER class."""
    from datetime import datetime
    import time
    start = time.time()
    start_time = "2024-01-01 08:00:00"
    end_time = "2024-01-21 08:00:00"
    
    format_time = "%Y-%m-%d %H:%M:%S"

    spider = IODA_SPIDER()
    spider.set_config(datetime.strptime(start_time, format_time), datetime.strptime(end_time, format_time))
    spider.run("events")

    end = time.time()

    print(f"用时{int(end - start)}秒。")

    
    for idx, item in enumerate(spider.events_data["data"]):
        print(idx, item)
